{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-a1ee041365f7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mplatform\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'{}{}-{}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mget_abbr_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mget_impl_ver\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mget_abi_tag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mcuda_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetoutput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"ldconfig -p|grep cudart.so|sed -e 's/.*\\\\.\\\\([0-9]*\\\\)\\\\.\\\\([0-9]*\\\\)$/cu\\\\1\\\\2/'\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0maccelerator\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcuda_output\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mexists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/dev/nvidia0'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m'cpu'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "from os.path import exists\n",
    "from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag\n",
    "platform = '{}{}-{}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag())\n",
    "cuda_output = !ldconfig -p|grep cudart.so|sed -e 's/.*\\.\\([0-9]*\\)\\.\\([0-9]*\\)$/cu\\1\\2/'\n",
    "accelerator = cuda_output[0] if exists('/dev/nvidia0') else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exists('/dev/nvidia0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "cuda_output = !ldconfig -p|grep cudart.so|sed -e 's/.*\\.\\([0-9]*\\)\\.\\([0-9]*\\)$/cu\\1\\2/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cuda_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtype = torch.FloatTensor\n",
    "# S: Symbol that shows starting of decoding input\n",
    "# E: Symbol that shows starting of decoding output\n",
    "# P: Symbol that will fill in blank sequence if current batch data size is short than time steps\n",
    "sentences = ['ich mochte ein bier P', 'S i want a beer', 'i want a beer E']\n",
    "\n",
    "# Transformer Parameters\n",
    "# Padding Should be Zero\n",
    "src_vocab = {'P' : 0, 'ich' : 1, 'mochte' : 2, 'ein' : 3, 'bier' : 4}\n",
    "src_vocab_size = len(src_vocab)\n",
    "\n",
    "tgt_vocab = {'P' : 0, 'i' : 1, 'want' : 2, 'a' : 3, 'beer' : 4, 'S' : 5, 'E' : 6}\n",
    "number_dict = {i: w for i, w in enumerate(tgt_vocab)}\n",
    "tgt_vocab_size = len(tgt_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "src_len = 5\n",
    "tgt_len = 5\n",
    "\n",
    "d_model = 512  # Embedding Size\n",
    "d_ff = 2048 # FeedForward dimension\n",
    "d_k = d_v = 64  # dimension of K(=Q), V\n",
    "n_layers = 6  # number of Encoder of Decoder Layer\n",
    "n_heads = 8  # number of heads in Multi-Head Attention\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_batch(sentences):\n",
    "    input_batch = [[src_vocab[n] for n in sentences[0].split()]]\n",
    "    output_batch = [[tgt_vocab[n] for n in sentences[1].split()]]\n",
    "    target_batch = [[tgt_vocab[n] for n in sentences[2].split()]]\n",
    "    return Variable(torch.LongTensor(input_batch)), Variable(torch.LongTensor(output_batch)), Variable(torch.LongTensor(target_batch))\n",
    "\n",
    "def get_sinusoid_encoding_table(n_position, d_model):\n",
    "    def cal_angle(position, hid_idx):\n",
    "        return position / np.power(10000, 2 * (hid_idx // 2) / d_model)\n",
    "    def get_posi_angle_vec(position):\n",
    "        return [cal_angle(position, hid_j) for hid_j in range(d_model)]\n",
    "\n",
    "    sinusoid_table = np.array([get_posi_angle_vec(pos_i) for pos_i in range(n_position)])\n",
    "    sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2])  # dim 2i\n",
    "    sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2])  # dim 2i+1\n",
    "    return torch.FloatTensor(sinusoid_table)\n",
    "\n",
    "def get_attn_pad_mask(seq_q, seq_k):\n",
    "    batch_size, len_q = seq_q.size()\n",
    "    batch_size, len_k = seq_k.size()\n",
    "    # eq(zero) is PAD token\n",
    "    pad_attn_mask = seq_k.data.eq(0).unsqueeze(1)  # batch_size x 1 x len_k(=len_q), one is masking\n",
    "    return pad_attn_mask.expand(batch_size, len_q, len_k)  # batch_size x len_q x len_k\n",
    "\n",
    "def get_attn_subsequent_mask(seq):\n",
    "    attn_shape = [seq.size(0), seq.size(1), seq.size(1)]\n",
    "    subsequent_mask = np.triu(np.ones(attn_shape), k=1)\n",
    "    subsequent_mask = torch.from_numpy(subsequent_mask).byte()\n",
    "    return subsequent_mask\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ScaledDotProductAttention(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(ScaledDotProductAttention, self).__init__()\n",
    "\n",
    "    def forward(self, Q, K, V, attn_mask):\n",
    "        scores = torch.matmul(Q, K.transpose(-1, -2)) / np.sqrt(d_k) # scores : [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n",
    "        scores.masked_fill_(attn_mask, -1e9) # Fills elements of self tensor with value where mask is one.\n",
    "        attn = nn.Softmax(dim=-1)(scores)\n",
    "        context = torch.matmul(attn, V)\n",
    "        return context, attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiHeadAttention(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MultiHeadAttention, self).__init__()\n",
    "        self.W_Q = nn.Linear(d_model, d_k * n_heads)\n",
    "        self.W_K = nn.Linear(d_model, d_k * n_heads)\n",
    "        self.W_V = nn.Linear(d_model, d_v * n_heads)\n",
    "    def forward(self, Q, K, V, attn_mask):\n",
    "        # q: [batch_size x len_q x d_model], k: [batch_size x len_k x d_model], v: [batch_size x len_k x d_model]\n",
    "        residual, batch_size = Q, Q.size(0)\n",
    "        # (B, S, D) -proj-> (B, S, D) -split-> (B, S, H, W) -trans-> (B, H, S, W)\n",
    "        q_s = self.W_Q(Q).view(batch_size, -1, n_heads, d_k).transpose(1,2)  # q_s: [batch_size x n_heads x len_q x d_k]\n",
    "        k_s = self.W_K(K).view(batch_size, -1, n_heads, d_k).transpose(1,2)  # k_s: [batch_size x n_heads x len_k x d_k]\n",
    "        v_s = self.W_V(V).view(batch_size, -1, n_heads, d_v).transpose(1,2)  # v_s: [batch_size x n_heads x len_k x d_v]\n",
    "\n",
    "        attn_mask = attn_mask.unsqueeze(1).repeat(1, n_heads, 1, 1) # attn_mask : [batch_size x n_heads x len_q x len_k]\n",
    "\n",
    "        # context: [batch_size x n_heads x len_q x d_v], attn: [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n",
    "        context, attn = ScaledDotProductAttention()(q_s, k_s, v_s, attn_mask)\n",
    "        context = context.transpose(1, 2).contiguous().view(batch_size, -1, n_heads * d_v) # context: [batch_size x len_q x n_heads * d_v]\n",
    "        output = nn.Linear(n_heads * d_v, d_model)(context)\n",
    "        return nn.LayerNorm(d_model)(output + residual), attn # output: [batch_size x len_q x d_model]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PoswiseFeedForwardNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(PoswiseFeedForwardNet, self).__init__()\n",
    "        self.conv1 = nn.Conv1d(in_channels=d_model, out_channels=d_ff, kernel_size=1)\n",
    "        self.conv2 = nn.Conv1d(in_channels=d_ff, out_channels=d_model, kernel_size=1)\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        residual = inputs # inputs : [batch_size, len_q, d_model]\n",
    "        output = nn.ReLU()(self.conv1(inputs.transpose(1, 2)))\n",
    "        output = self.conv2(output).transpose(1, 2)\n",
    "        return nn.LayerNorm(d_model)(output + residual)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EncoderLayer(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(EncoderLayer, self).__init__()\n",
    "        self.enc_self_attn = MultiHeadAttention()\n",
    "        self.pos_ffn = PoswiseFeedForwardNet()\n",
    "\n",
    "    def forward(self, enc_inputs, enc_self_attn_mask):\n",
    "        enc_outputs, attn = self.enc_self_attn(enc_inputs, enc_inputs, enc_inputs, enc_self_attn_mask) # enc_inputs to same Q,K,V\n",
    "        enc_outputs = self.pos_ffn(enc_outputs) # enc_outputs: [batch_size x len_q x d_model]\n",
    "        return enc_outputs, attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DecoderLayer(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(DecoderLayer, self).__init__()\n",
    "        self.dec_self_attn = MultiHeadAttention()\n",
    "        self.dec_enc_attn = MultiHeadAttention()\n",
    "        self.pos_ffn = PoswiseFeedForwardNet()\n",
    "\n",
    "    def forward(self, dec_inputs, enc_outputs, dec_self_attn_mask, dec_enc_attn_mask):\n",
    "        dec_outputs, dec_self_attn = self.dec_self_attn(dec_inputs, dec_inputs, dec_inputs, dec_self_attn_mask)\n",
    "        dec_outputs, dec_enc_attn = self.dec_enc_attn(dec_outputs, enc_outputs, enc_outputs, dec_enc_attn_mask)\n",
    "        dec_outputs = self.pos_ffn(dec_outputs)\n",
    "        return dec_outputs, dec_self_attn, dec_enc_attn\n",
    "\n",
    "class Encoder(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.src_emb = nn.Embedding(src_vocab_size, d_model)\n",
    "        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(src_len+1, d_model),freeze=True)\n",
    "        self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])\n",
    "\n",
    "    def forward(self, enc_inputs): # enc_inputs : [batch_size x source_len]\n",
    "        enc_outputs = self.src_emb(enc_inputs) + self.pos_emb(torch.LongTensor([[1,2,3,4,0]]))\n",
    "        enc_self_attn_mask = get_attn_pad_mask(enc_inputs, enc_inputs)\n",
    "        enc_self_attns = []\n",
    "        for layer in self.layers:\n",
    "            enc_outputs, enc_self_attn = layer(enc_outputs, enc_self_attn_mask)\n",
    "            enc_self_attns.append(enc_self_attn)\n",
    "        return enc_outputs, enc_self_attns\n",
    "\n",
    "class Decoder(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.tgt_emb = nn.Embedding(tgt_vocab_size, d_model)\n",
    "        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(tgt_len+1, d_model),freeze=True)\n",
    "        self.layers = nn.ModuleList([DecoderLayer() for _ in range(n_layers)])\n",
    "\n",
    "    def forward(self, dec_inputs, enc_inputs, enc_outputs): # dec_inputs : [batch_size x target_len]\n",
    "        dec_outputs = self.tgt_emb(dec_inputs) + self.pos_emb(torch.LongTensor([[5,1,2,3,4]]))\n",
    "        dec_self_attn_pad_mask = get_attn_pad_mask(dec_inputs, dec_inputs)\n",
    "        dec_self_attn_subsequent_mask = get_attn_subsequent_mask(dec_inputs)\n",
    "        dec_self_attn_mask = torch.gt((dec_self_attn_pad_mask + dec_self_attn_subsequent_mask), 0)\n",
    "\n",
    "        dec_enc_attn_mask = get_attn_pad_mask(dec_inputs, enc_inputs)\n",
    "\n",
    "        dec_self_attns, dec_enc_attns = [], []\n",
    "        for layer in self.layers:\n",
    "            dec_outputs, dec_self_attn, dec_enc_attn = layer(dec_outputs, enc_outputs, dec_self_attn_mask, dec_enc_attn_mask)\n",
    "            dec_self_attns.append(dec_self_attn)\n",
    "            dec_enc_attns.append(dec_enc_attn)\n",
    "        return dec_outputs, dec_self_attns, dec_enc_attns\n",
    "\n",
    "class Transformer(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Transformer, self).__init__()\n",
    "        self.encoder = Encoder()\n",
    "        self.decoder = Decoder()\n",
    "        self.projection = nn.Linear(d_model, tgt_vocab_size, bias=False)\n",
    "    def forward(self, enc_inputs, dec_inputs):\n",
    "        enc_outputs, enc_self_attns = self.encoder(enc_inputs)\n",
    "        dec_outputs, dec_self_attns, dec_enc_attns = self.decoder(dec_inputs, enc_inputs, enc_outputs)\n",
    "        dec_logits = self.projection(dec_outputs) # dec_logits : [batch_size x src_vocab_size x tgt_vocab_size]\n",
    "        return dec_logits.view(-1, dec_logits.size(-1)), enc_self_attns, dec_self_attns, dec_enc_attns\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0001 cost = 1.737171\n",
      "Epoch: 0002 cost = 0.618163\n",
      "Epoch: 0003 cost = 0.395703\n",
      "Epoch: 0004 cost = 0.040572\n",
      "Epoch: 0005 cost = 0.031891\n",
      "Epoch: 0006 cost = 0.017690\n",
      "Epoch: 0007 cost = 0.009949\n",
      "Epoch: 0008 cost = 0.015362\n",
      "Epoch: 0009 cost = 0.002595\n",
      "Epoch: 0010 cost = 0.001407\n",
      "Epoch: 0011 cost = 0.001537\n",
      "Epoch: 0012 cost = 0.002106\n",
      "Epoch: 0013 cost = 0.001013\n",
      "Epoch: 0014 cost = 0.000750\n",
      "Epoch: 0015 cost = 0.000506\n",
      "Epoch: 0016 cost = 0.000214\n",
      "Epoch: 0017 cost = 0.000394\n",
      "Epoch: 0018 cost = 0.000168\n",
      "Epoch: 0019 cost = 0.000211\n",
      "Epoch: 0020 cost = 0.000156\n",
      "ich mochte ein bier P -> ['i', 'want', 'a', 'beer', 'E']\n",
      "first head of last state enc_self_attns\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeoAAAIACAYAAABNWi9DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAXSUlEQVR4nO3de7Sld13f8c83M4Fw1WUuDYRAJIIFCtg4EBG5lSwDcS0XCxGrQIsUEhArFy1qFUQti8VNLg2IUwOhmqgUbUvAitAkFREIAZaUpgpyDQHM5FJICOb66x97DzluTpKZM+ec57v3vF5r7TXnPHufk+951sm85/c8z967xhgBAHo6ZOoBAIBbJtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYULPUqupnqur/VNU1VXXv+bZfqqonTz0bwGYQapZWVT0/ya8m2Z2k1tx1SZKfnWQogE0m1CyzZyd51hjj9UluWLP9Y0keMM1IAJtLqFlm90ryyXW2X5/kDts8C8CWEGqW2WeTnLDO9lOSXLTNswBsiZ1TDwAH4NVJTq+qO2Z2jvphVfW0JC9K8oxJJwPYJDXGmHoG2LCqelZmF5QdO990SZKXjjHOmG4qgM0j1KyEqjoiySFjjEunngVgMzlHzdKqqnOr6juTZIxx2d5IV9Vdq+rcaacD2BxW1CytqropydGLq+iqOirJJWOMQ6eZDGDzuJiMpVNVa6/0flBVXbHm8x1JTs7sXDXA0rOiZunMV9J7f3FrnYd8M8m/HWO8ZfumAtgaVtQso+/OLNCfTfLQJHvW3HddkkvHGDdOMRjAZrOiBoDGrKhZalV1bJJHJDkqC89iGGP81iRDAWwiK2qWVlU9JclbMntDjj25+bx1kowxxr0nGQxgEwk1S6uqPpPkj5K82DlpYFUJNUurqq5O8qAxxmenngVgq3hlMpbZnyY5ceohALaSi8lYKlX1xDWfvjfJK6rqAUn+d2bvQ/0tY4w/2c7ZALaCQ98slfmLneyLMcbYsaXDAGwDoQaAxpyjBoDGhJqlVVVvqaqfX2f7C6vqd6eYCWCzCTXL7JQk673v9Lnz+6CVqtpZVadU1eFTz8LyEGqW2XcmuXqd7d9I8l3bPAvcpjHGDUn+JMldpp6F5SHULLNPZf2V848k+bttngX21V8n+Z6ph2B5eB41y+w1Sd5cVUfl5kPgj03y/CTPnWwquHUvTfKaqvq1JB/N7AjQt4wxrphiKPry9CyWWlWdluRXkxwz33RJkpeNMd483VRwyxZeC2DtX8AVz/9nHULNSqiqIzP7fb506lng1lTVo27t/jHG/9quWVgOQs3Sq6p7J7l/ZquTi8YYn5t4pJVQVXdM8n1Z/72+vTwrbBPnqFlaVXXXJGck+bEkN928uf44yb8ZY1w12XBLrqpOSvIHSdZ7GtFI4vDsAaiqByY5LcnxSZ4xxvhKVT0hyRfGGB+fdjq6cdX3NqmqO1bVD1bVE6rqiWtvU8+2xF6f5EFJHpPkDvPbY+fbXjfhXKvg9UneneQeY4xDFm4ifQCq6oeTfCSz6yr+RWa/t8ks2r821Vz05dD3Nrit1Ym/+Damqi5P8oQxxvsXtj8yyX8dY3hRiQ2qqm9k9l7fn5l6llVTVR9O8rYxxpuq6qokDx5jfLaqvj/JOWOMu088Is1YUW8Pq5OtcYckl6+z/Yokh23zLKvmA0m+d+ohVtQDMnsv9UVXxAv1sA7nqLfHcUl+dIzx5akHWTEfSPKbVfW0McY1SVJVd0ry60n+atLJlt+bk7y6qu6e9d/r+2OTTLUarszssPfnF7afkORL2z4N7Qn19ti7OnEYcXO9IMmfJbmkqj6R2UVOD05yTZIfnnKwFfCO+Z+717nPxWQH5uwkr6qqJ2e2L3fOn7L16iRvnXQyWnKOeotU1QlrPj0uyX9I8luxOtlUVXWHJE9Jcr/MXjDioiRnjTG+OelgS66q7nVr948xvrBds6yaqjo0yZlJ/mVmv7M3zf88O8nTxxg3TjcdHQn1Fpm/+tDI7H/AW+NisgNQVUcn+cGs/1zfN00yFOyDqjo+yT/P7Pf242OMT088Ek0J9Ra5rRXJWlYnG1NVT03yu5n9Y+jK/OOXYxyunt0/86cKnjPGuP62njboBU9g+wg1S6uqvpDkbUl+Y/72gRyA+VGgo8cYly68HvUiR4H2U1W9IckvjzG+Mf/4Fo0xfm6bxmJJuJhsG1TVy5JcvPhGEVX17CTHjDFePM1kS++uSc4U6c0xxjhkvY/ZFA9Mcuiaj2+JlRPfxop6G1TVF5P8+BjjwwvbH5LkHWOMfT5Mzs2q6vQkfzvG+I9Tz7KKqurxmb1d6L2TnDzGuLiqnpnkc2OM/zntdKuhqu6cJGOMq6eehb78q3l7HJVkzzrbL0/yT7Z5llXywiSPr6r/VlW/WVUvWXuberhlVlVPSfL2JJ9O8t25eTW4I8mLppprVVTV8+f/gP9akq9V1cVV9YKquq2LT1nH/CWa31hVl1TVpVV1dlUdMfVcm8Wh7+3xxSSPSPLZhe2PjBc4OBCnJXlcksuSfE8WLiZL8htTDLUiXpTkWWOMP5yvovf6UOzXA1JVr0xyapJXJfngfPPDkrwkyd3iH0Ib8etJnp7krCTfTPJTSX47yY9PONOmEert8TtJXltVt0ty7nzbY5O8PMkrJptq+b04yc+PMV479SAr6D65OSJrXZ3ZtQFs3DOTPHOM8Y41286tqr/N7O8Kod5/T8zsHfP+MEmq6qwkH6iqHavwvHSh3gZjjNfMD8O8IcntMns60bWZvQb4q6acbcntSPLOqYdYUV9Oct8ki08dfGS8wt5m+MQtbHM6cmOOTfKtN+cZY1xQVTckuXuSiyebapP4pdgmY4xfTnJEkh+Y344cY/zScDXfgXhrZq9KxubbneQNVfXw+efHVtW/TvLKzA4psnH/ObOL9BY9J8nvbfMsq2JHkusWtt2QFVmMrsQP0VFVvTPJU8cYX59/vN5jkiRjjB/dztlWyB2TPLOqTs5sNbL40qyej7pBY4xXVtV3JHlvZu9Edl5mR4FePcZ446TDLaGF507vTPLU+e/th+bbTsxs9XfWds+2IirJ71fVtWu2HZbkP1XVNXs3LOvftUK9dS7PzRc3rfdWjBy4+yX5+Pzjf7pwnyMVB2iM8Svz1wC4f2ZH3y7yNKINW3zu9Efnf+59auZX57fF32P2zdvW2fb72z7FFvE8agBozDlqAGhMqLdZVZ069Qyryr7dOvbt1rFvt86q7Fuh3n4r8YvTlH27dezbrWPfbp2V2LdCDQCNrcTFZEd8145x3LGH3vYDG9hz+Y058nDvELgV7Nuts2z7dizRRf+XXX5Tjjh8edZMn/7EnaYeYZ9dn2tzaG4/9Rj77KpcedkY48jF7Svx9Kzjjj00F7zn2KnHAJq4cdza22lzIE455oSpR1hZ7xvvWHwlwCQOfQNAa0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY61DXVVnVtW7pp4DAKayc+oBbsPzktTUQwDAVFqHeozxtalnAIApOfQNAI21DjUAHOyWNtRVdWpVXVhVF+65/MapxwGALbG0oR5j7B5j7Bpj7Dry8B1TjwMAW2JpQw0ABwOhBoDGhBoAGhNqAGis+wuePH3qGQBgSlbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA01jLUVXV+VZ0+9RwAMLWWoQYAZm4z1FX1+Kq6qqp2zj+/T1WNqvrtNY95WVW9t6p2VNUZVfW5qvpmVX26ql5UVYeseeyZVfWuqnpeVV1SVVdW1Vur6o5770/yqCTPnf93RlUdt8k/NwAshZ378Jj3Jzksya4kH0ry6CSXJXnMmsc8OsmfZhb+S5I8OcmeJA9NsjvJ5UnOWPP4RyT5SpKTkhyb5O1JPpXk5Umel+S+Sf4myb+fP37Pfv5cALASbnNFPca4OsnHcnOYH53k9CT3qqq7zVfCD0ly/hjj+jHGS8YYHxljfH6M8fYkb07ykwvf9utJnjPG+L9jjD9P8l+SPHb+3/takuuSXDPG+Or8duPiXFV1alVdWFUX7rn82+4GgJWwr+eoz88s0MnssPT/SHLBfNvDk1w//zxV9ex5QPdU1dVJXpDkngvf76Ixxg1rPv9ykqP2Z/Axxu4xxq4xxq4jD9+xP18KAEtjf0L98Kq6f5K7JPnofNtjMov1X40xrq+qn0jyuiRnJjk5yfcleVOS2y18v+sXPh/7MQsAHDT25Rx1MjtPffskL0ryl2OMG6vq/MzOP1+a2fnpJPmhJB8eY3zrqVVVdfwG5rouiWUyAAe9fVrFrjlP/dQk5803fzCzC8FOzGx1ncwuCDthfqX4farqxZkdKt9fn0/y0Ko6rqqOWHvVOAAcTPYngOdltso9P0nGGP+Q2VXg12Z+fjrJ72R2BffZST6S5Lgkr9nAXK/ObFV9UWZXfC+e4waAg0KNMaae4YDtevBh44L3HDv1GEATN46bph5hZZ1yzAlTj7Cy3jfe8dExxq7F7Q4pA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNtQp1VT2uqt5fVVdW1RVV9Z6qut/UcwHAVFqFOsmdkrwuyUOTPDrJ15KcU1W3m3IoAJjKzqkHWGuM8cdrP6+qn07y9czC/ZcL952a5NQkuecxrX4MANg0rVbUVXV8VZ1dVZ+pqq8n+fvMZrzn4mPHGLvHGLvGGLuOPHzHts8KANuh21L0nCSXJDlt/ucNSS5K4tA3AAelNqGuqsOT3C/Jc8cY5823nZBGMwLAdusUwSuTXJbkWVV1cZJjkrwqs1U1AByU2pyjHmPclOQnkjwoySeTvDHJi5NcO+VcADClTivqjDHOTfLPFjbfeYpZAKCDNitqAODbCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANDYfoW6qs6vqtO3ahgA4B+zogaAxtqHuqpuN/UMADCVjYR6Z1W9vqqunN9eVVWHJLOoVtUrqupLVfWNqvpIVZ289our6v5V9e6quqqqLq2qP6iqo9fcf2ZVvauqfrGqvpTkSwf2IwLA8tpIqJ8y/7qHJTktyalJnj+/761JHpXkp5I8MMnbkpxTVQ9Okqq6W5K/SPLJJA9NclKSOyd5597Yzz0qyYOSPC7JYzcwIwCshJ0b+JqvJPm5McZI8jdVdd8kL6yq/57kJ5McN8b44vyxp1fVSZkF/WeSPCfJX48xfnHvN6uqf5XkiiS7klww3/wPSZ4xxrj2loaoqlMz+0dC7nnMRn4MAOhvIyvqD80jvdcHkxyT5IeSVJKLqurqvbckP5Lk+Pljvz/JIxfuv3h+3/Frvucnby3SSTLG2D3G2DXG2HXk4Ts28GMAQH+bvRQdSR6S5PqF7d+c/3lIkncn+YV1vvbv13z8jU2eCwCW0kZCfWJV1ZpV9Q8k+XJmK+tKcvQY47xb+NqPJXlyki+MMRZjDgAs2Mih77sneV1VfW9VPSnJv0vy2jHGp5KcleTMqnpSVd27qnZV1S9U1RPnX/vGJN+R5I+q6sT5Y06qqt1VdZdN+YkAYIVsZEV9VpIdST6c2aHuM5K8dn7fTyf5lSSvTHKPzC4SuyDJeUkyxvhyVT08ycuT/FmSw5J8McmfJ7nVc9IAcDDar1CPMR695tOfXef+65O8dH67pe/x6SRPupX7n74/MwHAKmv/ymQAcDATagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDG2oS6qs6sqrHO7UNTzwYAU9k59QAL3pfkaQvbrptiEADooFuorx1jfHXqIQCgizaHvgGAb9ct1I+rqqsXbq9Y74FVdWpVXVhVF+65/MbtnhMAtkW3Q99/keTUhW3/b70HjjF2J9mdJLsefNjY4rkAYBLdQn3NGOPvph4CALrodugbAFij24r69lV19MK2G8cYeyaZBgAm1i3UJyX5ysK2S5LcY4JZAGBybQ59jzGePsaodW4iDcBBq02oAYBvJ9QA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI3VGGPqGQ5YVe1J8oWp59hHRyS5bOohVpR9u3Xs261j326dZdu39xpjHLm4cSVCvUyq6sIxxq6p51hF9u3WsW+3jn27dVZl3zr0DQCNCTUANCbU22/31AOsMPt269i3W8e+3TorsW+dowaAxqyoAaAxoQaAxoQaABoTagBoTKgBoLH/D3L8rOuHaDniAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "first head of last state dec_self_attns\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeoAAAIACAYAAABNWi9DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAX3ElEQVR4nO3deZCkd33f8c9Xu+KQOAxIRAgEAhkcIByBRRgwV6SyAFe5KAI4NpAAAQmMYw472I4B44OiuMwRgWFjQCQWtgl2Eg7HNlhSjLkFlAmRbTCHEAKs0yAh0PnLH92Lxs1I2p2dmefbva9XVdfOPN0z+s5To33v73me7q4xRgCAng6aegAA4LoJNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbULLWq+tmq+n9VdVlV3WW+7Zer6olTzwawGYSapVVVz0vyoiS7k9Sau85N8nOTDAWwyYSaZfasJM8cY7w+yVVrtn86yT2nGQlgcwk1y+xOST63zvYrk9x0m2cB2BJCzTL7UpL7rbP9MUnO2uZZALbEzqkHgP3w6iQnV9UhmZ2jflBVPSXJC5M8fdLJADZJjTGmngE2rKqemdkFZUfNN52b5KVjjLdONxXA5hFqVkJVHZbkoDHGeVPPArCZnKNmaVXVaVX1Q0kyxrhgT6Sr6hZVddq00wFsDitqllZVXZPkiMVVdFXdNsm5Y4yDp5kMYPO4mIylU1Vrr/S+d1VdtObzHUlOyOxcNcDSs6Jm6cxX0nt+cWudh3w3yX8YY7xt+6YC2BpW1CyjO2cW6C8lOTbJ+WvuuyLJeWOMq6cYDGCzWVEDQGNW1Cy1qjoqyUOT3DYLz2IYY/z2JEMBbCIrapZWVT0pydsye0OO83PteeskGWOMu0wyGMAmEmqWVlV9MckfJnmxc9LAqhJqllZVXZrk3mOML009C8BW8cpkLLM/SfLAqYcA2EouJmOpVNXj1nz6gSSvqKp7Jvm/mb0P9feNMf54O2cD2AoOfbNU5i92sjfGGGPHlg4DsA2EGgAac44aABoTapZWVb2tqn5hne0vqKrfnWImgM0m1CyzxyRZ732nT5vfB61U1c6qekxV3WbqWVgeQs0y+6Ekl66z/TtJbr3Ns8ANGmNcleSPk9x86llYHkLNMvt81l85/0SSv9/mWWBv/XWSH556CJaH51GzzF6T5M1Vddtcewj8uCTPS/KcyaaC6/fSJK+pql9L8qnMjgB93xjjoimGoi9Pz2KpVdVJSV6U5PbzTecmedkY483TTQXXbeG1ANb+BVzx/H/WIdSshKo6PLPf5/OmngWuT1U9/PruH2P8n+2aheUg1Cy9qrpLkntktjo5a4zx5YlHWglVdUiS+2b99/r28qywTZyjZmlV1S2SvDXJv05yzbWb64+S/PsxxiWTDbfkqur4JL+fZL2nEY0kDs/uh6q6V5KTkhyT5OljjG9U1WOTnD3G+My009GNq763SVUdUlUPrqrHVtXj1t6mnm2JvT7JvZM8MslN57fj5tteN+Fcq+D1Sd6f5A5jjIMWbiK9H6rqx5N8MrPrKv5VZr+3ySzavzbVXPTl0Pc2uKHVib/4NqaqLkzy2DHGhxa2PyzJ/xhjeFGJDaqq72T2Xt9fnHqWVVNVH0/yjjHGm6rqkiT3GWN8qarun+S9Y4wjJx6RZqyot4fVyda4aZIL19l+UZKbbPMsq+bDSX5k6iFW1D0zey/1RRfFC/WwDueot8fRSX5yjPH1qQdZMR9O8ptV9ZQxxmVJUlWHJvn1JB+ZdLLl9+Ykr66qI7P+e31/epKpVsPFmR32/srC9vsl+dq2T0N7Qr099qxOHEbcXM9P8qdJzq2qz2Z2kdN9klyW5MenHGwFvHv+5+517nMx2f55Z5JXVdUTM9uXO+dP2Xp1krdPOhktOUe9Rarqfms+PTrJbyX57VidbKqqummSJyW5e2YvGHFWklPHGN+ddLAlV1V3ur77xxhnb9csq6aqDk5ySpJ/k9nv7DXzP9+Z5KljjKunm46OhHqLzF99aGT2P+D1cTHZfqiqI5I8OOs/1/dNkwwFe6GqjknyLzP7vf3MGOMLE49EU0K9RW5oRbKW1cnGVNWTk/xuZv8Yujj/9OUYh6tn9838qYLvHWNceUNPG/SCJ7B9hJqlVVVnJ3lHkt+Yv30g+2F+FOiIMcZ5C69HvchRoH1UVW9I8itjjO/MP75OY4yf36axWBIuJtsGVfWyJOcsvlFEVT0rye3HGC+eZrKld4skp4j05hhjHLTex2yKeyU5eM3H18XKiR9gRb0NquqrSZ4wxvj4wvYHJHn3GGOvD5Nzrao6OcnfjTH+89SzrKKqenRmbxd6lyQnjDHOqapnJPnyGOMvpp1uNVTVzZJkjHHp1LPQl381b4/bJjl/ne0XJvln2zzLKnlBkkdX1f+sqt+sqpesvU093DKrqicleVeSLyS5c65dDe5I8sKp5loVVfW8+T/gv5XkW1V1TlU9v6pu6OJT1jF/ieY3VtW5VXVeVb2zqg6beq7N4tD39vhqkocm+dLC9ofFCxzsj5OSPCrJBUl+OAsXkyX5jSmGWhEvTPLMMcYfzFfRe3ws9ut+qapXJjkxyauSfHS++UFJXpLkdvEPoY349SRPTXJqku8m+Zkkv5PkCRPOtGmEenu8Jclrq+pGSU6bbzsuycuTvGKyqZbfi5P8whjjtVMPsoLummsjstalmV0bwMY9I8kzxhjvXrPttKr6u8z+rhDqffe4zN4x7w+SpKpOTfLhqtqxCs9LF+ptMMZ4zfwwzBuS3CizpxNdntlrgL9qytmW3I4k75l6iBX19SR3S7L41MGHxSvsbYbPXsc2pyM35qgk339znjHGJ6rqqiRHJjlnsqk2iV+KbTLG+JUkhyX50fnt8DHGLw9X8+2Pt2f2qmRsvt1J3lBVD5l/flRV/bskr8zskCIb918zu0hv0bOT/LdtnmVV7EhyxcK2q7Iii9GV+CE6qqr3JHnyGOPb84/Xe0ySZIzxk9s52wo5JMkzquqEzFYjiy/N6vmoGzTGeGVV3TLJBzJ7J7LTMzsK9OoxxhsnHW4JLTx3emeSJ89/bz823/bAzFZ/p273bCuikvxeVV2+ZttNkvyXqrpsz4Zl/btWqLfOhbn24qb13oqR/Xf3JJ+Zf/zPF+5zpGI/jTF+df4aAPfI7OjbWZ5GtGGLz53+1PzPPU/N/Ob8tvh7zN55xzrbfm/bp9ginkcNAI05Rw0AjQn1NquqE6eeYVXZt1vHvt069u3WWZV9K9TbbyV+cZqyb7eOfbt17NutsxL7VqgBoLGVuJjssFvvGEcfdfANP7CB8y+8OoffZnneIfDznz1k6hH22pW5PAfnxlOPsZLs261j326dZdu3l+TiC8YYhy9uX4mnZx191MH5xJ8dNfUYK+mEI+879QgAB4QPjncvvhJgEoe+AaA1oQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaCx1qGuqlOq6n1TzwEAU9k59QA34LlJauohAGAqrUM9xvjW1DMAwJQc+gaAxlqHGgAOdEsb6qo6sarOrKozz7/w6qnHAYAtsbShHmPsHmPsGmPsOvw2O6YeBwC2xNKGGgAOBEINAI0JNQA0JtQA0Fj3Fzx56tQzAMCUrKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBobOfUA2yGv7nsVjn2M0+YeoyVdPVJh009wso67C0fnXoEYAlYUQNAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA01jLUVXVGVZ089RwAMLWWoQYAZm4w1FX16Kq6pKp2zj+/a1WNqvqdNY95WVV9oKp2VNVbq+rLVfXdqvpCVb2wqg5a89hTqup9VfXcqjq3qi6uqrdX1SF77k/y8CTPmf93RlUdvck/NwAshZ178ZgPJblJkl1JPpbkEUkuSPLINY95RJI/ySz85yZ5YpLzkxybZHeSC5O8dc3jH5rkG0mOT3JUkncl+XySlyd5bpK7JfnbJP9p/vjz9/HnAoCVcIMr6jHGpUk+nWvD/IgkJye5U1Xdbr4SfkCSM8YYV44xXjLG+OQY4ytjjHcleXOSn174tt9O8uwxxt+MMf48yX9Pctz8v/etJFckuWyM8c357erFuarqxKo6s6rOvOrbl23kZweA9vb2HPUZmQU6mR2W/t9JPjHf9pAkV84/T1U9ax7Q86vq0iTPT3LHhe931hjjqjWffz3Jbfdl8DHG7jHGrjHGrp23OGRfvhQAlsa+hPohVXWPJDdP8qn5tkdmFuuPjDGurKqfSvK6JKckOSHJfZO8KcmNFr7flQufj32YBQAOGHtzjjqZnae+cZIXJvmrMcbVVXVGZuefz8vs/HSS/FiSj48xvv/Uqqo6ZgNzXZFkxwa+DgBWyl6tYtecp35yktPnmz+a2YVgD8xsdZ3MLgi73/xK8btW1YszO1S+r76S5NiqOrqqDlt71TgAHEj2JYCnZ7bKPSNJxhjfy+wq8MszPz+d5C2ZXcH9ziSfTHJ0ktdsYK5XZ7aqPiuzK74Xz3EDwAGhxhhTz7DfDr3r7cbd3/C0qcdYSVe/57CpR1hZh73lo1OPADTywfHuT40xdi1ud0gZABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxnZOPcBmuOaSnbn8g4dPPcZKuuo2U0+wuq467v5Tj7Cydv7Fp6YeATaNFTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA01irUVfWoqvpQVV1cVRdV1Z9V1d2nngsAptIq1EkOTfK6JMcmeUSSbyV5b1XdaMqhAGAqO6ceYK0xxh+t/byqnpbk25mF+68W7jsxyYlJcvDNb7VdIwLAtmq1oq6qY6rqnVX1xar6dpJ/yGzGOy4+doyxe4yxa4yxa8chh277rACwHVqtqJO8N8m5SU6a/3lVkrOSOPQNwAGpTair6jZJ7p7kOWOM0+fb7pdGMwLAdusUwYuTXJDkmVV1TpLbJ3lVZqtqADggtTlHPca4JslPJbl3ks8leWOSFye5fMq5AGBKnVbUGWOcluRfLGy+2RSzAEAHbVbUAMAPEmoAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMZ2Tj3AZhgHJVceMvUUq+nO7zh76hFW1xhTT7CyvvyiB089wso66rc+MvUIBxwragBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoLF9CnVVnVFVJ2/VMADAP2VFDQCNtQ91Vd1o6hkAYCobCfXOqnp9VV08v72qqg5KZlGtqldU1deq6jtV9cmqOmHtF1fVParq/VV1SVWdV1W/X1VHrLn/lKp6X1X9UlV9LcnX9u9HBIDltZFQP2n+dQ9KclKSE5M8b37f25M8PMnPJLlXknckeW9V3SdJqup2Sf4yyeeSHJvk+CQ3S/KePbGfe3iSeyd5VJLjNjAjAKyEnRv4mm8k+fkxxkjyt1V1tyQvqKr/leSnkxw9xvjq/LEnV9XxmQX9Z5M8O8lfjzF+ac83q6p/m+SiJLuSfGK++XtJnj7GuPy6hqiqEzP7R0J23uJWG/gxAKC/jayoPzaP9B4fTXL7JD+WpJKcVVWX7rkl+Ykkx8wfe/8kD1u4/5z5fces+Z6fu75IJ8kYY/cYY9cYY9eOQw/dwI8BAP1tZEV9fUaSByS5cmH7d+d/HpTk/Ul+cZ2v/Yc1H39nk+cCgKW0kVA/sKpqzar6R5N8PbOVdSU5Yoxx+nV87aeTPDHJ2WOMxZgDAAs2cuj7yCSvq6ofqarHJ/mPSV47xvh8klOTnFJVj6+qu1TVrqr6xap63Pxr35jklkn+sKoeOH/M8VW1u6puvik/EQCskI2sqE9NsiPJxzM71P3WJK+d3/e0JL+a5JVJ7pDZRWKfSHJ6kowxvl5VD0ny8iR/muQmSb6a5M+TXO85aQA4EO1TqMcYj1jz6c+tc/+VSV46v13X9/hCksdfz/1P3ZeZAGCVtX9lMgA4kAk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjO6ceYDPc+B+vzp3ed/HUY6yka25986lHWFnfO9K+3Sp19dQTrK6dR99x6hFW15fX32xFDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANBYm1BX1SlVNda5fWzq2QBgKjunHmDBB5M8ZWHbFVMMAgAddAv15WOMb049BAB00ebQNwDwg7qF+lFVdenC7RXrPbCqTqyqM6vqzCuuumy75wSAbdHt0PdfJjlxYds/rvfAMcbuJLuT5JaHHDm2eC4AmES3UF82xvj7qYcAgC66HfoGANbotqK+cVUdsbDt6jHG+ZNMAwAT6xbq45N8Y2HbuUnuMMEsADC5Noe+xxhPHWPUOjeRBuCA1SbUAMAPEmoAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMZqjDH1DPutqs5PcvbUc+ylw5JcMPUQK8q+3Tr27daxb7fOsu3bO40xDl/cuBKhXiZVdeYYY9fUc6wi+3br2Ldbx77dOquybx36BoDGhBoAGhPq7bd76gFWmH27dezbrWPfbp2V2LfOUQNAY1bUANCYUANAY0INAI0JNQA0JtQA0Nj/B/vnwh/8ux/uAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "first head of last state dec_enc_attns\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeoAAAIACAYAAABNWi9DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAXMUlEQVR4nO3de7Sld13f8c83M4EQAV0moSEhEIlggXJpHEBEbk2WAVzLxULEKtAihQTEykWLWgVRy2IBQS4NiFOBUA0qjbblYkVokooIhABLSlMFuYYAZnJpIBdz49c/9h5y3JwkM2fOOc9373m91tprznn2Piff86yTec/veZ69d40xAgD0dMjUAwAAt0yoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaFmqVXVz1TV/6mqa6rqnvNtv1RVT556NoDNINQsrap6fpJfTbI7Sa256+IkPzvJUACbTKhZZs9O8qwxxuuS3Lhm+8eT3G+akQA2l1CzzO6R5FPrbL8hyR22eRaALSHULLPPJTlxne2PT3LhNs8CsCV2Tj0AHIDTk5xRVYdndo76YVX1tCQvSvKMSScD2CQ1xph6BtiwqnpWZheUHTffdHGSl44x3jzdVACbR6hZCVV1ZJJDxhiXTD0LwGZyjpqlVVXnVNV3JckY49K9ka6qO1fVOdNOB7A5rKhZWlX1zSRHL66iq+ouSS4eYxw6zWQAm8fFZCydqlp7pfcDquryNZ/vSHJKZueqAZaeFTVLZ76S3vuLW+s85Nok/3aM8Zbtmwpga1hRs4y+J7NAfy7JQ5LsWXPf9UkuGWPcNMVgAJvNihoAGrOiZqlV1XFJHpHkLll4FsMY47cmGQpgE1lRs7Sq6ilJ3pLZG3Lsyc3nrZNkjDHuOclgAJtIqFlaVfXZJH+U5MXOSQOrSqhZWlV1VZIHjDE+N/UsAFvFK5OxzP40yUOnHgJgK7mYjKVSVU9c8+n7kryiqu6X5H9n9j7U3zLG+JPtnA1gKzj0zVKZv9jJvhhjjB1bOgzANhBqAGjMOWoAaEyoWVpV9Zaq+vl1tr+wqn53ipkANptQs8wen2S9950+Z34ftFJVO6vq8VV1xNSzsDyEmmX2XUmuWmf71Um+e5tngds0xrgxyZ8kudPUs7A8hJpl9umsv3L+kSR/t82zwL766yTfO/UQLA/Po2aZvTrJm6rqLrn5EPhJSZ6f5LmTTQW37qVJXl1Vv5bkY5kdAfqWMcblUwxFX56exVKrqtOS/GqSY+ebLk7ysjHGm6abCm7ZwmsBrP0LuOL5/6xDqFkJVXVUZr/Pl0w9C9yaqnrUrd0/xvhf2zULy0GoWXpVdc8k981sdXLhGOPzE4+0Eqrq8CQPyvrv9e3lWWGbOEfN0qqqOyd5c5IfS/LNmzfXHyf5N2OMb0w23JKrqpOT/EGS9Z5GNJI4PHsAqur+SU5LckKSZ4wxvlpVT0jyxTHGJ6adjm5c9b1NqurwqvrBqnpCVT1x7W3q2ZbY65I8IMljktxhfjtpvu21E861Cl6X5D1J7jbGOGThJtIHoKp+OMlHM7uu4l9k9nubzKL9a1PNRV8OfW+D21qd+ItvY6rqsiRPGGN8YGH7I5P81zGGF5XYoKq6OrP3+v7s1LOsmqr6SJK3jTHeWFXfSPLAMcbnqur7k7xrjHHMxCPSjBX19rA62Rp3SHLZOtsvT3LYNs+yaj6Y5PumHmJF3S+z91JfdHm8UA/rcI56exyf5EfHGF+ZepAV88Ekv1lVTxtjXJMkVfUdSX49yV9NOtnye1OS06vqmKz/Xt8fn2Sq1XBFZoe9v7Cw/cQkX972aWhPqLfH3tWJw4ib6wVJ/izJxVX1ycwucnpgkmuS/PCUg62As+d/7l7nPheTHZi3J3lVVT05s325c/6UrdOTvHXSyWjJOeotUlUnrvn0+CT/IclvxepkU1XVHZI8Jcl9MnvBiAuTnDXGuHbSwZZcVd3j1u4fY3xxu2ZZNVV1aJIzk/zLzH5nvzn/8+1Jnj7GuGm66ehIqLfI/NWHRmb/A94aF5MdgKo6OskPZv3n+r5xkqFgH1TVCUn+eWa/t58YY3xm4pFoSqi3yG2tSNayOtmYqnpqkt/N7B9DV+QfvxzjcPXs/pk/VfBdY4wbbutpg17wBLaPULO0quqLSd6W5Dfmbx/IAZgfBTp6jHHJwutRL3IUaD9V1euT/PIY4+r5x7dojPFz2zQWS8LFZNugql6W5KLFN4qoqmcnOXaM8eJpJlt6d05ypkhvjjHGIet9zKa4f5JD13x8S6yc+DZW1Nugqr6U5MfHGB9Z2P7gJGePMfb5MDk3q6ozkvztGOM/Tj3LKqqqx2X2dqH3THLKGOOiqnpmks+PMf7ntNOthqq6Y5KMMa6aehb68q/m7XGXJHvW2X5Zkn+yzbOskhcmeVxV/beq+s2qesna29TDLbOqekqSdyT5TJLvyc2rwR1JXjTVXKuiqp4//wf8lUmurKqLquoFVXVbF5+yjvlLNL+hqi6uqkuq6u1VdeTUc20Wh763x5eSPCLJ5xa2PzJe4OBAnJbksUkuTfK9WbiYLMlvTDHUinhRkmeNMf5wvore68OxXw9IVb0yyalJXpXkQ/PND0vykiR3jX8IbcSvJ3l6krOSXJvkp5L8dpIfn3CmTSPU2+N3krymqm6X5Jz5tpOSvDzJKyabavm9OMnPjzFeM/UgK+heuTkia12V2bUBbNwzkzxzjHH2mm3nVNXfZvZ3hVDvvydm9o55f5gkVXVWkg9W1Y5VeF66UG+DMcar54dhXp/kdpk9nei6zF4D/FVTzrbkdiR559RDrKivJLl3ksWnDj4yXmFvM3zyFrY5HbkxxyX51pvzjDHOr6obkxyT5KLJptokfim2yRjjl5McmeQH5rejxhi/NFzNdyDemtmrkrH5did5fVU9fP75cVX1r5O8MrNDimzcf87sIr1Fz0nye9s8y6rYkeT6hW03ZkUWoyvxQ3RUVe9M8tQxxtfnH6/3mCTJGONHt3O2FXJ4kmdW1SmZrUYWX5rV81E3aIzxyqr6ziTvy+ydyM7N7CjQ6WOMN0w63BJaeO70ziRPnf/efni+7aGZrf7O2u7ZVkQl+f2qum7NtsOS/KequmbvhmX9u1aot85lufnipvXeipEDd58kn5h//E8X7nOk4gCNMX5l/hoA983s6NuFnka0YYvPnf7Y/M+9T8382vy2+HvMvnnbOtt+f9un2CKeRw0AjTlHDQCNCfU2q6pTp55hVdm3W8e+3Tr27dZZlX0r1NtvJX5xmrJvt459u3Xs262zEvtWqAGgsZW4mOzI794xjj/u0Nt+YAN7LrspRx3hHQK3gn27dezbrbNs+/bTnzx86hH22Q25Lofm9lOPsc++kSsuHWMctbh9JZ6edfxxh+b89x439RgAK++UYx409Qgr6/3j7MVXAkzi0DcAtCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANNY61FV1ZlW9e+o5AGAqO6ce4DY8L0lNPQQATKV1qMcYV049AwBMyaFvAGisdagB4GC3tKGuqlOr6oKqumDPZTdNPQ4AbImlDfUYY/cYY9cYY9dRR+yYehwA2BJLG2oAOBgINQA0JtQA0JhQA0Bj3V/w5OlTzwAAU7KiBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgsZahrqrzquqMqecAgKm1DDUAMHOboa6qx1XVN6pq5/zze1XVqKrfXvOYl1XV+6pqR1W9uao+X1XXVtVnqupFVXXImseeWVXvrqrnVdXFVXVFVb21qg7fe3+SRyV57vy/M6rq+E3+uQFgKezch8d8IMlhSXYl+XCSRye5NMlj1jzm0Un+NLPwX5zkyUn2JHlIkt1JLkvy5jWPf0SSryY5OclxSd6R5NNJXp7keUnuneRvkvz7+eP37OfPBQAr4TZX1GOMq5J8PDeH+dFJzkhyj6q663wl/OAk540xbhhjvGSM8dExxhfGGO9I8qYkP7nwbb+e5DljjP87xvjzJP8lyUnz/96VSa5Pcs0Y42vz202Lc1XVqVV1QVVdsOeyb7sbAFbCvp6jPi+zQCezw9L/I8n5820PT3LD/PNU1bPnAd1TVVcleUGSuy98vwvHGDeu+fwrSe6yP4OPMXaPMXaNMXYddcSO/flSAFga+xPqh1fVfZPcKcnH5tsek1ms/2qMcUNV/USS1yY5M8kpSR6U5I1Jbrfw/W5Y+HzsxywAcNDYl3PUyew89e2TvCjJX44xbqqq8zI7/3xJZuenk+SHknxkjPGtp1ZV1QkbmOv6JJbJABz09mkVu+Y89VOTnDvf/KHMLgR7aGar62R2QdiJ8yvF71VVL87sUPn++kKSh1TV8VV15NqrxgHgYLI/ATw3s1XueUkyxviHzK4Cvy7z89NJfiezK7jfnuSjSY5P8uoNzHV6ZqvqCzO74nvxHDcAHBRqjDH1DAds1wMPG+e/97ipxwBYeacc86CpR1hZ7x9nf2yMsWtxu0PKANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjrUJdVY+tqg9U1RVVdXlVvbeq7jP1XAAwlVahTvIdSV6b5CFJHp3kyiTvqqrbTTkUAExl59QDrDXG+OO1n1fVTyf5embh/suF+05NcmqS3P3YVj8GAGyaVivqqjqhqt5eVZ+tqq8n+fvMZrz74mPHGLvHGLvGGLuOOmLHts8KANuh21L0XUkuTnLa/M8bk1yYxKFvAA5KbUJdVUckuU+S544xzp1vOzGNZgSA7dYpglckuTTJs6rqoiTHJnlVZqtqADgotTlHPcb4ZpKfSPKAJJ9K8oYkL05y3ZRzAcCUOq2oM8Y4J8k/W9h8xylmAYAO2qyoAYBvJ9QA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0Bj+xXqqjqvqs7YqmEAgH/MihoAGmsf6qq63dQzAMBUNhLqnVX1uqq6Yn57VVUdksyiWlWvqKovV9XVVfXRqjpl7RdX1X2r6j1V9Y2quqSq/qCqjl5z/5lV9e6q+sWq+nKSLx/YjwgAy2sjoX7K/OseluS0JKcmef78vrcmeVSSn0py/yRvS/KuqnpgklTVXZP8RZJPJXlIkpOT3DHJO/fGfu5RSR6Q5LFJTtrAjACwEnZu4Gu+muTnxhgjyd9U1b2TvLCq/nuSn0xy/BjjS/PHnlFVJ2cW9J9J8pwkfz3G+MW936yq/lWSy5PsSnL+fPM/JHnGGOO6Wxqiqk7N7B8JufuxG/kxAKC/jayoPzyP9F4fSnJskh9KUkkurKqr9t6S/EiSE+aP/f4kj1y4/6L5fSes+Z6furVIJ8kYY/cYY9cYY9dRR+zYwI8BAP1t9lJ0JHlwkhsWtl87//OQJO9J8gvrfO3fr/n46k2eCwCW0kZC/dCqqjWr6h9I8pXMVtaV5Ogxxrm38LUfT/LkJF8cYyzGHABYsJFD38ckeW1VfV9VPSnJv0vymjHGp5OcleTMqnpSVd2zqnZV1S9U1RPnX/uGJN+Z5I+q6qHzx5xcVbur6k6b8hMBwArZyIr6rCQ7knwks0Pdb07ymvl9P53kV5K8MsndMrtI7Pwk5ybJGOMrVfXwJC9P8mdJDkvypSR/nuRWz0kDwMFov0I9xnj0mk9/dp37b0jy0vntlr7HZ5I86Vbuf/r+zAQAq6z9K5MBwMFMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaE2oAaEyoAaAxoQaAxoQaABoTagBoTKgBoDGhBoDGhBoAGhNqAGhMqAGgMaEGgMaEGgAaaxPqqjqzqsY6tw9PPRsATGXn1AMseH+Spy1su36KQQCgg26hvm6M8bWphwCALtoc+gYAvl23UD+2qq5auL1ivQdW1alVdUFVXbDnspu2e04A2BbdDn3/RZJTF7b9v/UeOMbYnWR3kux64GFji+cCgEl0C/U1Y4y/m3oIAOii26FvAGCNbivq21fV0Qvbbhpj7JlkGgCYWLdQn5zkqwvbLk5ytwlmAYDJtTn0PcZ4+hij1rmJNAAHrTahBgC+nVADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADQm1ADQmFADQGNCDQCNCTUANCbUANCYUANAY0INAI0JNQA0JtQA0JhQA0BjQg0AjQk1ADRWY4ypZzhgVbUnyRennmMfHZnk0qmHWFH27daxb7eOfbt1lm3f3mOMcdTixpUI9TKpqgvGGLumnmMV2bdbx77dOvbt1lmVfevQNwA0JtQA0JhQb7/dUw+wwuzbrWPfbh37duusxL51jhoAGrOiBoDGhBoAGhNqAGhMqAGgMaEGgMb+P65LpOu+tN9PAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = Transformer()\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "def showgraph(attn):\n",
    "    attn = attn[-1].squeeze(0)[0]\n",
    "    attn = attn.squeeze(0).data.numpy()\n",
    "    fig = plt.figure(figsize=(n_heads, n_heads)) # [n_heads, n_heads]\n",
    "    ax = fig.add_subplot(1, 1, 1)\n",
    "    ax.matshow(attn, cmap='viridis')\n",
    "    ax.set_xticklabels(['']+sentences[0].split(), fontdict={'fontsize': 14}, rotation=90)\n",
    "    ax.set_yticklabels(['']+sentences[2].split(), fontdict={'fontsize': 14})\n",
    "    plt.show()\n",
    "\n",
    "for epoch in range(20):\n",
    "    optimizer.zero_grad()\n",
    "    enc_inputs, dec_inputs, target_batch = make_batch(sentences)\n",
    "    outputs, enc_self_attns, dec_self_attns, dec_enc_attns = model(enc_inputs, dec_inputs)\n",
    "    loss = criterion(outputs, target_batch.contiguous().view(-1))\n",
    "    print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "# Test\n",
    "predict, _, _, _ = model(enc_inputs, dec_inputs)\n",
    "predict = predict.data.max(1, keepdim=True)[1]\n",
    "print(sentences[0], '->', [number_dict[n.item()] for n in predict.squeeze()])\n",
    "\n",
    "print('first head of last state enc_self_attns')\n",
    "showgraph(enc_self_attns)\n",
    "\n",
    "print('first head of last state dec_self_attns')\n",
    "showgraph(dec_self_attns)\n",
    "\n",
    "print('first head of last state dec_enc_attns')\n",
    "showgraph(dec_enc_attns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
